package com.otaku.crawler;

import java.util.List;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.otaku.crawler.model.MTimePojo;
import com.otaku.crawler.model.NASaveMoneyPojo;
import com.otaku.crawler.parser.MTimeParser;
import com.otaku.crawler.parser.NASaveMoneyParser;
import com.otaku.crawler.util.Constants;
import com.otaku.crawler.util.EmailHelper;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;


/**
 * crawler for cloudknows(Mtime,movie)
 * @author zzd
 *
 */
public class MTimeCrawler extends WebCrawler{
	private static Logger log = Logger.getRootLogger();
	
	public boolean shouldVisit(WebURL url) {
		String href = url.getURL().toLowerCase();
		
		if(Constants.FILTERS.matcher(href).matches()
				|| !href.startsWith("http://theater.mtime.com/china")
				|| href.contains("/movie/")
				|| href.contains("/cinema/")
				|| href.contains("/ecoupon/")
				|| href.contains("/giftcard/")
				|| href.contains("/discount/")
				|| href.endsWith(".html")
				|| href.contains("?"))
			return false;
		
		int num = 0;
		for(char c : href.toCharArray()){
			if(c == '/') num++;
		}
		
		if(num<=5){
			log.info("should visit:"+href);
			return true;
		}else{
			log.info("should not visit:"+href);
			return false;
		}
	}
	
	public void visit(Page page) {
		String url = page.getWebURL().getURL();
		try{
			String urls[] = url.split("/");
			if(!urls[urls.length-1].matches(Constants.NUM_REGIX)) {
				log.warn("visited :"+url+" but not interested.please check");
				return;
			}
			log.info("visiting:"+url);
			
			if (page.getParseData() instanceof HtmlParseData) {
				HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
				String html = htmlParseData.getHtml();

				Document d = Jsoup.parse(html);
				MTimePojo nal = MTimeParser.parse(d,url);

				if(nal!=null) 
					log.info("\nlist size:" + nal.getFilmInfos().size());
				else
					log.warn("\nurl:"+url+" got no filminfo");
			}
		}catch(Exception e){
			EmailHelper.sendMsg("MTimeCrawler caught exception in url:\n"
					+ url, e);
			log.error("MTimeCrawler cught exception", e);
			return;
		}
	}
}
